1 module epsilon.ell1gen; 2 3 import core.time : MonoTime; 4 import EAG = epsilon.eag; 5 import EmitGen = epsilon.emitgen; 6 import Shift = epsilon.shift; 7 import EvalGen = epsilon.slaggen; 8 import epsilon.settings; 9 import io : Input, Position, read; 10 import log; 11 import runtime; 12 import std.bitmanip : BitArray; 13 import std.conv : to; 14 import std.format; 15 import std.stdio; 16 import std.typecons; 17 18 private const nil = 0; 19 private const endTok = 0; 20 private const undefTok = 1; 21 private const sepTok = 2; 22 private const firstUserTok = 3; 23 private enum nElemsPerSET = size_t.sizeof * 8; 24 private const firstEdge = 1; 25 private const firstGenSet = 1; 26 private const firstGenSetT = 1; 27 28 private struct NontRecord 29 { 30 BitArray First; 31 BitArray Follow; 32 BitArray IniFollow; 33 EAG.Alt DefaultAlt; 34 int Edge; 35 int AltRec; 36 int OptRec; 37 int AltExp; 38 int OptExp; 39 int FirstIndex; 40 int FollowIndex; 41 bool Anonym; 42 } 43 44 private struct AltRecord 45 { 46 BitArray Dir; 47 } 48 49 private struct FactorRecord 50 { 51 int Rec; 52 } 53 54 private struct EdgeRecord 55 { 56 int Dest; 57 int Next; 58 } 59 60 private NontRecord[] Nont; 61 private AltRecord[] Alt; 62 private FactorRecord[] Factor; 63 private EdgeRecord[] Edge; 64 private int NextEdge; 65 private BitArray[] GenSet; 66 private int NextGenSet; 67 private BitArray[] GenSetT; 68 private int NextGenSetT; 69 private BitArray TestNonts; 70 private BitArray GenNonts; 71 private BitArray RegNonts; 72 private BitArray ConflictNonts; 73 private int nToks; 74 public bool Error; 75 private bool Warning; 76 private bool UseReg; 77 78 public void Test(Settings settings) 79 in (EAG.Performed(EAG.analysed | EAG.predicates)) 80 { 81 info!"ELL(1) testing %s"(EAG.BaseName); 82 EAG.History &= ~EAG.parsable; 83 Init(settings); 84 scope (exit) 85 Finit; 86 if (!GrammarOk) 87 return; 88 ComputeDir; 89 if (Error || Warning) 90 return; 91 info!"%s grammar is ELL(1)"(EAG.BaseName); 92 EAG.History |= EAG.parsable; 93 } 94 95 public string Generate(Settings settings) 96 in (EAG.Performed(EAG.analysed | EAG.predicates | EAG.isSLAG)) 97 { 98 info!"ELL(1) writing %s"(EAG.BaseName); 99 EAG.History &= ~EAG.parsable; 100 Init(settings); 101 scope (exit) 102 Finit; 103 if (!GrammarOk) 104 assert(0, "TODO: error handling for parser generator"); 105 ComputeDir; 106 if (Error) 107 assert(0, "TODO: error handling for parser generator"); 108 ComputeDefaultAlts; 109 ComputeSets; 110 111 const fileName = GenerateMod(No.parsePass, settings); 112 113 EAG.History |= EAG.parsable; 114 return fileName; 115 } 116 117 public string GenerateParser(Settings settings) 118 in (EAG.Performed(EAG.analysed | EAG.predicates | EAG.hasEvaluator)) 119 { 120 info!"ELL(1) writing parser of %s"(EAG.BaseName); 121 EAG.History &= ~EAG.parsable; 122 Init(settings); 123 scope (exit) 124 Finit; 125 if (!GrammarOk) 126 assert(0, "TODO: error handling for parser generator"); 127 EAG.History = 0; 128 Shift.Shift; 129 ComputeDir; 130 if (Error) 131 assert(0, "TODO: error handling for parser generator"); 132 ComputeDefaultAlts; 133 ComputeSets; 134 return GenerateMod(Yes.parsePass, settings); 135 } 136 137 private void Init(Settings settings) 138 { 139 int i; 140 141 nToks = EAG.NextHTerm - EAG.firstHTerm + firstUserTok; 142 if (EAG.NextHNont >= 1) 143 Nont = new NontRecord[EAG.NextHNont]; 144 else 145 Nont = new NontRecord[1]; 146 for (i = EAG.firstHNont; i < EAG.NextHNont; ++i) 147 { 148 Nont[i].First = BitArray(); 149 Nont[i].First.length = nToks + 1; 150 Nont[i].Follow = BitArray(); 151 Nont[i].Follow.length = nToks + 1; 152 Nont[i].IniFollow = BitArray(); 153 Nont[i].IniFollow.length = nToks + 1; 154 155 Nont[i].DefaultAlt = null; 156 Nont[i].AltRec = nil; 157 Nont[i].OptRec = nil; 158 Nont[i].AltExp = nil; 159 Nont[i].OptExp = nil; 160 Nont[i].FirstIndex = nil; 161 Nont[i].FollowIndex = nil; 162 Nont[i].Anonym = EAG.All[i] && EAG.HNont[i].anonymous; 163 } 164 if (EAG.NextHAlt >= 1) 165 Alt = new AltRecord[EAG.NextHAlt]; 166 else 167 Alt = new AltRecord[1]; 168 for (i = EAG.firstHAlt; i < EAG.NextHAlt; ++i) 169 { 170 Alt[i].Dir = BitArray(); 171 Alt[i].Dir.length = nToks + 1; 172 } 173 if (EAG.NextHFactor >= 1) 174 Factor = new FactorRecord[EAG.NextHFactor]; 175 else 176 Factor = new FactorRecord[1]; 177 for (i = EAG.firstHFactor; i < EAG.NextHFactor; ++i) 178 Factor[i].Rec = nil; 179 Edge = new EdgeRecord[255]; 180 NextEdge = firstEdge; 181 GenSet = new BitArray[511]; 182 NextGenSet = firstGenSet; 183 GenSetT = new BitArray[255]; 184 NextGenSetT = firstGenSetT; 185 TestNonts = EAG.All - EAG.Pred; 186 GenNonts = EAG.Prod & EAG.Reach; 187 GenNonts -= EAG.Pred; 188 Error = false; 189 Warning = false; 190 UseReg = !settings.p; 191 RegNonts = BitArray(); 192 RegNonts.length = EAG.NextHNont + 1; 193 ConflictNonts = BitArray(); 194 ConflictNonts.length = EAG.NextHNont + 1; 195 if (UseReg) 196 ComputeRegNonts; 197 } 198 199 /** 200 * R whole procedure 201 */ 202 private void ComputeRegNonts() 203 { 204 EAG.Alt A; 205 EAG.Factor F; 206 207 void TraverseRegNonts(size_t N) 208 { 209 EAG.Alt A = EAG.HNont[N].Def.Sub; 210 211 do 212 { 213 EAG.Factor F = A.Sub; 214 215 while (F !is null) 216 { 217 if (auto nont = cast(EAG.Nont) F) 218 if (TestNonts[nont.Sym] && !RegNonts[nont.Sym]) 219 { 220 RegNonts[nont.Sym] = true; 221 TraverseRegNonts(nont.Sym); 222 } 223 F = F.Next; 224 } 225 A = A.Next; 226 } 227 while (A !is null); 228 } 229 230 void DeleteConflictNont(int N) 231 { 232 EAG.Alt A = EAG.HNont[N].Def.Sub; 233 234 ConflictNonts[N] = false; 235 do 236 { 237 for (EAG.Factor F = A.Sub; F !is null; F = F.Next) 238 if (auto nont = cast(EAG.Nont) F) 239 if (ConflictNonts[nont.Sym]) 240 DeleteConflictNont(nont.Sym); 241 A = A.Next; 242 } 243 while (A !is null); 244 } 245 246 RegNonts[] = false; 247 foreach (N; TestNonts.bitsSet) 248 if (TestNonts[N] && EAG.HNont[N].IsToken && !RegNonts[N]) 249 { 250 RegNonts[N] = true; 251 TraverseRegNonts(N); 252 } 253 ConflictNonts = RegNonts.dup; 254 foreach (N; ConflictNonts.bitsSet) 255 if (ConflictNonts[N]) 256 { 257 A = EAG.HNont[N].Def.Sub; 258 do 259 { 260 F = A.Last; 261 while (F !is null && cast(EAG.Nont) F && !TestNonts[(cast(EAG.Nont) F).Sym]) 262 F = F.Prev; 263 if (F !is null) 264 F = F.Prev; 265 while (F !is null) 266 { 267 if (auto nont = cast(EAG.Nont) F) 268 if (ConflictNonts[nont.Sym]) 269 DeleteConflictNont(nont.Sym); 270 F = F.Prev; 271 } 272 A = A.Next; 273 } 274 while (A !is null); 275 } 276 } 277 278 private void Finit() @nogc nothrow @safe 279 { 280 Nont = null; 281 Alt = null; 282 Factor = null; 283 Edge = null; 284 GenSet = null; 285 GenSetT = null; 286 } 287 288 /** 289 * R whole procedure 290 */ 291 private bool GrammarOk() 292 { 293 EAG.Alt A; 294 EAG.Factor F; 295 bool Ok = true; 296 297 if (UseReg) 298 { 299 if (RegNonts[EAG.StartSym]) 300 { 301 if (EAG.HNont[EAG.StartSym].IsToken) 302 error!"start symbol must not be a token"; 303 else 304 error!"start symbol must not be a sub-token"; 305 Ok = false; 306 } 307 foreach (N; TestNonts.bitsSet) 308 if (EAG.HNont[N].IsToken) 309 { 310 if (EAG.Null[N]) 311 { 312 error!"marked token %s is nullable"(EAG.HNontRepr(N)); 313 Ok = false; 314 } 315 if (Nont[N].Anonym) 316 { 317 fatal!"token in %s is anonymous"(EAG.NamedHNontRepr(N)); 318 Ok = false; 319 } 320 } 321 foreach (N; TestNonts.bitsSet) 322 if (!RegNonts[N]) 323 { 324 A = EAG.HNont[N].Def.Sub; 325 do 326 { 327 F = A.Sub; 328 while (F !is null) 329 { 330 if (auto nont = cast(EAG.Nont) F) 331 if (TestNonts[nont.Sym] && RegNonts[nont.Sym] && !EAG.HNont[nont.Sym].IsToken) 332 { 333 if (Nont[N].Anonym) 334 error!"nonterminal in %s calls sub-token %s"( 335 EAG.NamedHNontRepr(N), EAG.HNontRepr(nont.Sym)); 336 else 337 error!"nonterminal %s calls sub-token %s"( 338 EAG.HNontRepr(N), EAG.HNontRepr(nont.Sym)); 339 Ok = false; 340 } 341 F = F.Next; 342 } 343 A = A.Next; 344 } 345 while (A !is null); 346 } 347 } 348 return Ok; 349 } 350 351 private void ComputeDir() 352 { 353 EAG.Alt A; 354 EAG.Factor F; 355 int[] State; 356 size_t[] Stack; 357 int Top; 358 BitArray NullAlts; 359 BitArray Toks; 360 bool IsLast; 361 362 void ComputeFirst(size_t N) 363 { 364 int n; 365 int E; 366 size_t N1; 367 bool leftRecursion = false; 368 369 Stack[Top] = N; 370 ++Top; 371 n = Top; 372 State[N] = n; 373 E = Nont[N].Edge; 374 while (E != nil) 375 { 376 N1 = Edge[E].Dest; 377 if (N1 == N) 378 leftRecursion = true; 379 if (State[N1] == 0) 380 ComputeFirst(N1); 381 if (State[N1] < State[N]) 382 State[N] = State[N1]; 383 Nont[N].First |= Nont[N1].First; 384 E = Edge[E].Next; 385 } 386 if (State[N] == n) 387 { 388 string[] items; 389 390 leftRecursion = leftRecursion || Top > n; 391 do 392 { 393 --Top; 394 N1 = Stack[Top]; 395 State[N1] = int.max; 396 if (leftRecursion) 397 { 398 if (Nont[N1].Anonym) 399 { 400 items ~= format!"EBNF expression in %s\n%s" 401 (EAG.NamedHNontRepr(N1), EAG.HNont[N1].Def.Sub.Pos); 402 } 403 else 404 { 405 items ~= EAG.NamedHNontRepr(N1); 406 } 407 } 408 Nont[N1].First = Nont[N].First; 409 } 410 while (Top >= n); 411 if (leftRecursion) 412 { 413 error!"left recursion over nonterminals%-(\n%s%)"(items); 414 Error = true; 415 } 416 } 417 } 418 419 void ComputeFollow(size_t N) 420 { 421 int n; 422 int E; 423 size_t N1; 424 425 Stack[Top] = N; 426 ++Top; 427 n = Top; 428 State[N] = n; 429 E = Nont[N].Edge; 430 while (E != nil) 431 { 432 N1 = Edge[E].Dest; 433 if (State[N1] == 0) 434 ComputeFollow(N1); 435 if (State[N1] < State[N]) 436 State[N] = State[N1]; 437 Nont[N].Follow |= Nont[N1].Follow; 438 E = Edge[E].Next; 439 } 440 if (State[N] == n) 441 { 442 do 443 { 444 --Top; 445 N1 = Stack[Top]; 446 State[N1] = int.max; 447 Nont[N1].Follow = Nont[N].Follow; 448 } 449 while (Top >= n); 450 } 451 } 452 453 void Conflict(size_t N, Position Pos, BitArray Dir, BitArray PrevDirs) 454 { 455 import std.algorithm : map; 456 457 const msg = format!"director set conflict in %s: %-(%s, %)\n%s" 458 (EAG.NamedHNontRepr(N), (Dir & PrevDirs).bitsSet.map!TokRepr, Pos); 459 460 if ((Dir - PrevDirs).bitsSet.empty) 461 { 462 error!"%s\nalternative will never be chosen"(msg); 463 Error = true; 464 } 465 else 466 { 467 warn!"%s"(msg); 468 Warning = true; 469 } 470 } 471 472 State = new int[EAG.NextHNont]; 473 Stack = new size_t[EAG.NextHNont]; 474 Top = 0; 475 NullAlts = BitArray(); 476 NullAlts.length = EAG.NextHAlt; 477 Toks = BitArray(); 478 Toks.length = nToks + 1; 479 NextEdge = firstEdge; 480 for (size_t N = EAG.firstHNont; N < EAG.NextHNont; ++N) 481 { 482 Nont[N].Edge = nil; 483 State[N] = 0; 484 } 485 foreach (N; TestNonts.bitsSet) 486 { 487 Nont[N].First[] = false; 488 A = EAG.HNont[N].Def.Sub; 489 do 490 { 491 F = A.Sub; 492 while (true) 493 { 494 if (F is null) 495 break; 496 if (auto term = cast(EAG.Term) F) 497 { 498 Nont[N].First[term.Sym - EAG.firstHTerm + firstUserTok] = true; 499 break; 500 } 501 else if (auto nont = cast(EAG.Nont) F) 502 { 503 if (TestNonts[nont.Sym]) 504 { 505 NewEdge(N, nont.Sym); 506 if (!EAG.Null[nont.Sym]) 507 break; 508 } 509 } 510 F = F.Next; 511 } 512 A = A.Next; 513 } 514 while (A !is null); 515 } 516 foreach (N; TestNonts.bitsSet) 517 if (State[N] == 0) 518 ComputeFirst(N); 519 NextEdge = firstEdge; 520 for (size_t N = EAG.firstHNont; N < EAG.NextHNont; ++N) 521 { 522 Nont[N].Edge = nil; 523 Nont[N].Follow[] = false; 524 } 525 Nont[EAG.StartSym].Follow[endTok] = true; 526 NullAlts[] = false; 527 foreach (N; TestNonts.bitsSet) 528 { 529 A = EAG.HNont[N].Def.Sub; 530 do 531 { 532 if (cast(EAG.Rep) EAG.HNont[N].Def !is null) 533 Toks = Nont[N].First.dup; 534 else 535 Toks[] = false; 536 F = A.Last; 537 IsLast = true; 538 while (F !is null) 539 { 540 if (auto term = cast(EAG.Term) F) 541 { 542 Toks[] = false; 543 Toks[term.Sym - EAG.firstHTerm + firstUserTok] = true; 544 IsLast = false; 545 } 546 else if (auto nont = cast(EAG.Nont) F) 547 { 548 if (TestNonts[nont.Sym]) 549 { 550 if (IsLast) 551 NewEdge(nont.Sym, N.to!int); 552 Nont[nont.Sym].Follow |= Toks; 553 if (UseReg && !RegNonts[N] && RegNonts[nont.Sym]) 554 Nont[nont.Sym].Follow[sepTok] = true; 555 if (EAG.Null[nont.Sym]) 556 { 557 Toks |= Nont[nont.Sym].First; 558 } 559 else 560 { 561 Toks = Nont[nont.Sym].First.dup; 562 IsLast = false; 563 } 564 } 565 } 566 F = F.Prev; 567 } 568 if (IsLast) 569 NullAlts[A.Ind] = true; 570 A = A.Next; 571 } 572 while (A !is null); 573 } 574 foreach (N; TestNonts.bitsSet) 575 Nont[N].IniFollow = Nont[N].Follow.dup; 576 for (size_t N = EAG.firstHNont; N < EAG.NextHNont; ++N) 577 State[N] = 0; 578 foreach (N; TestNonts.bitsSet) 579 if (State[N] == 0) 580 ComputeFollow(N); 581 foreach (N; TestNonts.bitsSet) 582 { 583 Toks[] = false; 584 A = EAG.HNont[N].Def.Sub; 585 do 586 { 587 if (NullAlts[A.Ind]) 588 Alt[A.Ind].Dir = Nont[N].Follow.dup; 589 else 590 Alt[A.Ind].Dir[] = false; 591 F = A.Sub; 592 while (true) 593 { 594 if (F is null) 595 break; 596 if (auto term = cast(EAG.Term) F) 597 { 598 Alt[A.Ind].Dir[term.Sym - EAG.firstHTerm + firstUserTok] = true; 599 break; 600 } 601 else if (auto nont = cast(EAG.Nont) F) 602 { 603 if (TestNonts[nont.Sym]) 604 { 605 Alt[A.Ind].Dir |= Nont[nont.Sym].First; 606 if (!EAG.Null[nont.Sym]) 607 break; 608 } 609 } 610 F = F.Next; 611 } 612 if (!(Alt[A.Ind].Dir & Toks).bitsSet.empty) 613 { 614 Conflict(N, A.Pos, Alt[A.Ind].Dir, Toks); 615 Alt[A.Ind].Dir -= Toks; 616 } 617 Toks |= Alt[A.Ind].Dir; 618 A = A.Next; 619 } 620 while (A !is null); 621 if (cast(EAG.Opt) EAG.HNont[N].Def || cast(EAG.Rep) EAG.HNont[N].Def) 622 { 623 if (!(Nont[N].Follow & Toks).bitsSet.empty) 624 { 625 if (!UseReg || !ConflictNonts[N] || Toks[sepTok]) 626 { 627 if (auto opt = cast(EAG.Opt) EAG.HNont[N].Def) 628 Conflict(N, opt.EmptyAltPos, Nont[N].Follow, Toks); 629 else if (auto rep = cast(EAG.Rep) EAG.HNont[N].Def) 630 Conflict(N, rep.EmptyAltPos, Nont[N].Follow, Toks); 631 } 632 } 633 } 634 } 635 } 636 637 private string TokRepr(size_t Tok) @safe 638 { 639 if (Tok == endTok) 640 return "<end>"; 641 else if (Tok == undefTok) 642 return "<undef>"; 643 else if (Tok == sepTok) 644 return "<sep>"; 645 else 646 return EAG.HTermRepr(Tok.to!int + EAG.firstHTerm - firstUserTok); 647 } 648 649 private void ComputeDefaultAlts() 650 { 651 struct AltRecord 652 { 653 int Nont; 654 int Deg; 655 int Prio; 656 EAG.Alt Alt; 657 } 658 659 struct StackRecord 660 { 661 int Nont; 662 int APrio; 663 EAG.Alt Alt; 664 } 665 666 EAG.Alt A; 667 EAG.Factor F; 668 int E; 669 int APrio; 670 AltRecord[] Alt; 671 StackRecord[] Stack; 672 int Top; 673 int[] StackPos; 674 BitArray DefNonts; 675 676 void TestDeg(int AInd) 677 { 678 if (Alt[AInd].Deg == 0) 679 { 680 const N = Alt[AInd].Nont; 681 const i = StackPos[N]; 682 683 if (i == int.max) 684 { 685 Stack[Top].Nont = N; 686 Stack[Top].APrio = Alt[AInd].Prio; 687 Stack[Top].Alt = Alt[AInd].Alt; 688 StackPos[N] = Top; 689 ++Top; 690 } 691 else if (i >= 0 && Stack[i].APrio > Alt[AInd].Prio) 692 { 693 Stack[i].APrio = Alt[AInd].Prio; 694 Stack[i].Alt = Alt[AInd].Alt; 695 } 696 } 697 } 698 699 void Pop(ref int Edge) 700 { 701 int i; 702 int MinPrio; 703 int MinPos; 704 i = Top; 705 --Top; 706 MinPrio = int.max; 707 do 708 { 709 --i; 710 if (Stack[i].APrio < MinPrio) 711 { 712 MinPrio = Stack[i].APrio; 713 MinPos = i; 714 } 715 } 716 while (i != 0 && MinPrio != 1); 717 Nont[Stack[MinPos].Nont].DefaultAlt = Stack[MinPos].Alt; 718 Edge = Nont[Stack[MinPos].Nont].Edge; 719 StackPos[Stack[Top].Nont] = MinPos; 720 StackPos[Stack[MinPos].Nont] = -1; 721 Stack[MinPos] = Stack[Top]; 722 } 723 724 if (EAG.NextHAlt >= 1) 725 Alt = new AltRecord[EAG.NextHAlt]; 726 if (EAG.NextHNont >= 1) 727 Stack = new StackRecord[EAG.NextHNont]; 728 Top = 0; 729 if (EAG.NextHNont >= 1) 730 StackPos = new int[EAG.NextHNont]; 731 DefNonts = GenNonts.dup; 732 NextEdge = firstEdge; 733 for (size_t N = EAG.firstHNont; N < EAG.NextHNont; ++N) 734 { 735 Nont[N].Edge = nil; 736 Nont[N].DefaultAlt = null; 737 StackPos[N] = int.max; 738 if (GenNonts[N] && (cast(EAG.Opt) EAG.HNont[N].Def || cast(EAG.Rep) EAG.HNont[N].Def)) 739 DefNonts[N] = false; 740 } 741 foreach (N; DefNonts.bitsSet) 742 { 743 A = EAG.HNont[N].Def.Sub; 744 APrio = 1; 745 do 746 { 747 Alt[A.Ind].Nont = N.to!int; 748 Alt[A.Ind].Alt = A; 749 Alt[A.Ind].Deg = 0; 750 Alt[A.Ind].Prio = APrio; 751 F = A.Sub; 752 while (F !is null) 753 { 754 if (auto nont = cast(EAG.Nont) F) 755 if (DefNonts[nont.Sym]) 756 { 757 ++Alt[A.Ind].Deg; 758 NewEdge(nont.Sym, A.Ind); 759 } 760 F = F.Next; 761 } 762 TestDeg(A.Ind); 763 A = A.Next; 764 ++APrio; 765 } 766 while (A !is null); 767 } 768 while (Top > 0) 769 { 770 Pop(E); 771 while (E != nil) 772 { 773 --Alt[Edge[E].Dest].Deg; 774 TestDeg(Edge[E].Dest); 775 E = Edge[E].Next; 776 } 777 } 778 } 779 780 private void ComputeSets() 781 { 782 BitArray Start; 783 784 void NewGenSet(BitArray Toks, ref int GenSetIndex) 785 { 786 int i = firstGenSet; 787 788 while (i < NextGenSet && GenSet[i] != Toks) 789 ++i; 790 GenSetIndex = i; 791 if (i == NextGenSet) 792 { 793 if (NextGenSet >= GenSet.length) 794 Expand; 795 GenSet[NextGenSet] = Toks.dup; 796 ++NextGenSet; 797 } 798 } 799 800 void NewGenSetT(BitArray Toks, ref int GenSetTIndex) 801 { 802 int i = firstGenSetT; 803 804 while (i < NextGenSetT && GenSetT[i] != Toks) 805 ++i; 806 GenSetTIndex = i; 807 if (i == NextGenSetT) 808 { 809 if (NextGenSetT >= GenSetT.length) 810 Expand; 811 GenSetT[NextGenSetT] = Toks.dup; 812 ++NextGenSetT; 813 } 814 } 815 816 void ComputeRecoverySets(size_t N, ref BitArray LocalRec) 817 { 818 EAG.Alt A = EAG.HNont[N].Def.Sub; 819 const RealAlt = A.Next !is null; 820 EAG.Factor F; 821 BitArray S; 822 823 S.length = nToks + 1; 824 do 825 { 826 if (cast(EAG.Rep) EAG.HNont[N].Def) 827 S = LocalRec | Nont[N].First; 828 else 829 S = LocalRec.dup; 830 F = A.Last; 831 while (F !is null) 832 { 833 if (auto term = cast(EAG.Term) F) 834 { 835 S[term.Sym - EAG.firstHTerm + firstUserTok] = true; 836 NewGenSet(S, Factor[F.Ind].Rec); 837 } 838 else if (auto nont = cast(EAG.Nont) F) 839 { 840 if (GenNonts[nont.Sym]) 841 { 842 if (!Nont[nont.Sym].Anonym) 843 { 844 if (UseReg && !RegNonts[N] && RegNonts[nont.Sym]) 845 S[sepTok] = true; 846 NewGenSet(S, Factor[F.Ind].Rec); 847 S |= Nont[nont.Sym].First; 848 } 849 else 850 { 851 ComputeRecoverySets(nont.Sym, S); 852 } 853 } 854 } 855 F = F.Prev; 856 } 857 A = A.Next; 858 } 859 while (A !is null); 860 LocalRec |= Nont[N].First; 861 if (cast(EAG.Opt) EAG.HNont[N].Def || cast(EAG.Rep) EAG.HNont[N].Def) 862 NewGenSet(LocalRec, Nont[N].OptRec); 863 if (RealAlt) 864 NewGenSet(LocalRec, Nont[N].AltRec); 865 } 866 867 Start = BitArray(); 868 Start.length = nToks + 1; 869 foreach (N; GenNonts.bitsSet) 870 { 871 Start[] = false; 872 if (N == EAG.StartSym) 873 Start[endTok] = true; 874 if (!Nont[N].Anonym) 875 ComputeRecoverySets(N, Start); 876 if (cast(EAG.Opt) EAG.HNont[N].Def || cast(EAG.Rep) EAG.HNont[N].Def) 877 { 878 if (!Nont[N].Anonym) 879 { 880 NewGenSet(Nont[N].First, Nont[N].OptExp); 881 } 882 else 883 { 884 Start = Nont[N].First | Nont[N].IniFollow; 885 NewGenSet(Start, Nont[N].OptExp); 886 } 887 NewGenSetT(Nont[N].First, Nont[N].FirstIndex); 888 NewGenSetT(Nont[N].Follow, Nont[N].FollowIndex); 889 } 890 if (EAG.HNont[N].Def.Sub.Next !is null) 891 NewGenSet(Nont[N].First, Nont[N].AltExp); 892 } 893 } 894 895 private string GenerateMod(Flag!"parsePass" parsePass, Settings settings) 896 { 897 File output; 898 Input Fix; 899 int Tok; 900 BitArray AllToks; 901 string name; 902 long TabTimeStamp; 903 size_t loopCount; 904 905 void TraverseNont(size_t N, bool FirstNontCall, BitArray Poss) 906 { 907 bool ExactOneToken; 908 int TheOneToken; 909 910 void TraverseAlts(EAG.Alt A, bool FirstNontCall, BitArray Poss) 911 { 912 int Tok; 913 BitArray Toks; 914 bool FirstTok; 915 916 void TraverseFactors(EAG.Factor F, bool FirstNontCall, BitArray Poss) 917 { 918 bool TwoCalls = false; 919 BitArray Poss1 = Poss.dup; 920 921 while (F !is null) 922 { 923 if (auto term = cast(EAG.Term) F) 924 { 925 if (Poss1[term.Sym - EAG.firstHTerm + firstUserTok]) 926 { 927 Poss1[term.Sym - EAG.firstHTerm + firstUserTok] = false; 928 if (Poss1.bitsSet.empty) 929 { 930 output.writeln("S.Get(Tok);"); 931 output.writeln("IsRepairMode = false;"); 932 } 933 else 934 { 935 output.write("if (Tok != "); 936 output.write(term.Sym - EAG.firstHTerm + firstUserTok); 937 output.writeln(")"); 938 output.write("RecoveryTerminal("); 939 output.write(term.Sym - EAG.firstHTerm + firstUserTok); 940 output.write(", "); 941 output.write(Factor[F.Ind].Rec - firstGenSet); 942 output.writeln(");"); 943 output.writeln("else"); 944 output.writeln("{"); 945 output.writeln("S.Get(Tok);"); 946 output.writeln("IsRepairMode = false;"); 947 output.writeln("}"); 948 } 949 } 950 else 951 { 952 output.write("RecoveryTerminal("); 953 output.write(term.Sym - EAG.firstHTerm + firstUserTok); 954 output.write(", "); 955 output.write(Factor[F.Ind].Rec - firstGenSet); 956 output.writeln(");"); 957 } 958 Poss1 = AllToks.dup; 959 } 960 else if (auto nont = cast(EAG.Nont) F) 961 { 962 if (GenNonts[nont.Sym]) 963 { 964 EvalGen.GenSynPred(N, nont.Actual.Params); 965 if (!Nont[nont.Sym].Anonym) 966 { 967 if (FirstNontCall) 968 { 969 output.writeln("if (RecTop >= RecStack.length) ParserExpand;"); 970 FirstNontCall = false; 971 } 972 if (TwoCalls) 973 output.write("RecStack[RecTop - 1] = "); 974 else 975 output.write("RecStack[RecTop] = "); 976 output.write(Factor[F.Ind].Rec - firstGenSet, ";"); 977 if (!TwoCalls) 978 output.writeln("++RecTop;"); 979 if (UseReg && !RegNonts[N] && RegNonts[nont.Sym]) 980 output.writeln("S.Get = &S.Get3;"); 981 output.write("P", nont.Sym); 982 EvalGen.GenActualParams(nont.Actual.Params, true); 983 output.writeln("; // ", EAG.HNontRepr(nont.Sym)); 984 if (UseReg && !RegNonts[N] && RegNonts[nont.Sym]) 985 { 986 output.writeln("if (Tok == sepTok)"); 987 output.writeln("{"); 988 output.writeln("S.Get(Tok);"); 989 output.writeln("IsRepairMode = false;"); 990 output.writeln("}"); 991 output.writeln("S.Get = &S.Get2;"); 992 } 993 if (F.Next !is null && cast(EAG.Nont) F.Next 994 && GenNonts[(cast(EAG.Nont) F.Next).Sym] 995 && !Nont[(cast(EAG.Nont) F.Next).Sym].Anonym) 996 TwoCalls = true; 997 else 998 TwoCalls = false; 999 if (!TwoCalls) 1000 output.writeln("--RecTop;"); 1001 } 1002 else 1003 { 1004 TraverseNont(nont.Sym, FirstNontCall, Poss1); 1005 } 1006 EvalGen.GenAnalPred(N, nont.Actual.Params); 1007 Poss1 = AllToks.dup; 1008 } 1009 else if (EAG.Pred[nont.Sym]) 1010 { 1011 EvalGen.GenSynPred(N, nont.Actual.Params); 1012 EvalGen.GenPredCall(nont.Sym, nont.Actual.Params); 1013 EvalGen.GenAnalPred(N, nont.Actual.Params); 1014 } 1015 else 1016 { 1017 output.writeln(`throw new Exception("runtime error: call of nonproductive nonterminal!");`); 1018 warn!"generated compiler contains corrupt code for non-productive nonterminals"; 1019 Warning = true; 1020 } 1021 } 1022 F = F.Next; 1023 } 1024 } 1025 1026 if (A.Next is null) 1027 { 1028 EvalGen.InitScope(A.Scope); 1029 EvalGen.GenAnalPred(N, A.Formal.Params); 1030 TraverseFactors(A.Sub, FirstNontCall, Poss); 1031 if (cast(EAG.Rep) EAG.HNont[N].Def) 1032 EvalGen.GenRepAlt(N.to!int, A); 1033 else 1034 EvalGen.GenSynPred(N, A.Formal.Params); 1035 } 1036 else 1037 { 1038 if (!EAG.Null[N]) 1039 Toks = Nont[N].First.dup; 1040 else 1041 Toks = Nont[N].First | Nont[N].Follow; 1042 1043 const LoopNeeded = !(Poss <= Toks); 1044 const label = format!"loop%s"(loopCount); 1045 1046 if (LoopNeeded) 1047 { 1048 ++loopCount; 1049 output.writeln(label, ": while (1)"); 1050 output.writeln("{"); 1051 } 1052 output.writeln("switch (Tok)"); 1053 output.writeln("{"); 1054 do 1055 { 1056 if (!LoopNeeded && (Alt[A.Ind].Dir & Poss).bitsSet.empty) 1057 { 1058 warn!"dead alternative in %s\n%s"(EAG.NamedHNontRepr(N), A.Pos); 1059 Warning = true; 1060 } 1061 output.write("case "); 1062 FirstTok = true; 1063 // foreach (Tok; Alt[A.Ind].Dir) 1064 for (Tok = 0; Tok < nToks; ++Tok) 1065 { 1066 if (Alt[A.Ind].Dir[Tok]) 1067 { 1068 if (!FirstTok) 1069 { 1070 output.writeln(":"); 1071 output.write("case "); 1072 } 1073 output.write(Tok); 1074 FirstTok = false; 1075 } 1076 } 1077 output.writeln(":"); 1078 EvalGen.InitScope(A.Scope); 1079 EvalGen.GenAnalPred(N, A.Formal.Params); 1080 TraverseFactors(A.Sub, FirstNontCall, Alt[A.Ind].Dir); 1081 if (cast(EAG.Rep) EAG.HNont[N].Def) 1082 EvalGen.GenRepAlt(N.to!int, A); 1083 else 1084 EvalGen.GenSynPred(N, A.Formal.Params); 1085 if (LoopNeeded) 1086 output.writeln("break ", label, ";"); 1087 else 1088 output.writeln("break;"); 1089 A = A.Next; 1090 } 1091 while (A !is null); 1092 if (LoopNeeded) 1093 { 1094 A = Nont[N].DefaultAlt; 1095 output.writeln("default:"); 1096 output.writeln("if (IsRepairMode)"); 1097 output.writeln("{"); 1098 Toks = AllToks - Toks; 1099 EvalGen.InitScope(A.Scope); 1100 EvalGen.GenAnalPred(N, A.Formal.Params); 1101 TraverseFactors(A.Sub, FirstNontCall, Toks); 1102 if (cast(EAG.Rep) EAG.HNont[N].Def) 1103 EvalGen.GenRepAlt(N.to!int, A); 1104 else 1105 EvalGen.GenSynPred(N, A.Formal.Params); 1106 output.writeln("break ", label, ";"); 1107 output.writeln("}"); 1108 output.write("ErrorRecovery("); 1109 output.write(Nont[N].AltExp - firstGenSet); 1110 output.write(", "); 1111 output.write(Nont[N].AltRec - firstGenSet); 1112 output.writeln(");"); 1113 } 1114 else 1115 { 1116 output.writeln("default:"); 1117 output.writeln("assert(0);"); 1118 } 1119 output.writeln("}"); 1120 if (LoopNeeded) 1121 output.writeln("}"); 1122 } 1123 } 1124 1125 void TestOneToken(BitArray Toks, ref bool ExactOneToken, ref int TheOneToken) 1126 { 1127 int Tok = 0; 1128 1129 ExactOneToken = false; 1130 while (Tok < nToks) 1131 { 1132 if (Toks[Tok]) 1133 { 1134 if (ExactOneToken) 1135 { 1136 ExactOneToken = false; 1137 return; 1138 } 1139 ExactOneToken = true; 1140 TheOneToken = Tok; 1141 } 1142 ++Tok; 1143 } 1144 } 1145 1146 if (auto opt = cast(EAG.Opt) EAG.HNont[N].Def) 1147 { 1148 if (Poss <= Nont[N].Follow && (Nont[N].First & Poss).bitsSet.empty) 1149 { 1150 warn!"dead brackets in %s\n%s"(EAG.NamedHNontRepr(N), EAG.HNont[N].Def.Sub.Pos); 1151 Warning = true; 1152 } 1153 else if (Poss <= Nont[N].First) 1154 { 1155 warn!"useless brackets in %s\n%s"(EAG.NamedHNontRepr(N), EAG.HNont[N].Def.Sub.Pos); 1156 Warning = true; 1157 } 1158 output.writeln("while (1)"); 1159 output.writeln("{"); 1160 output.write("if ("); 1161 TestOneToken(Nont[N].First, ExactOneToken, TheOneToken); 1162 if (ExactOneToken) 1163 { 1164 output.write("Tok == ", TheOneToken); 1165 } 1166 else 1167 { 1168 output.write("SetT["); 1169 output.write(DIV(Nont[N].FirstIndex - firstGenSetT, nElemsPerSET)); 1170 output.write("][Tok] & 1uL << "); 1171 output.write(MOD(Nont[N].FirstIndex - firstGenSetT, nElemsPerSET)); 1172 } 1173 output.writeln(")"); 1174 output.writeln("{"); 1175 TraverseAlts(EAG.HNont[N].Def.Sub, FirstNontCall, Nont[N].First); 1176 output.writeln("break;"); 1177 output.writeln("}"); 1178 output.write("else if ("); 1179 TestOneToken(Nont[N].Follow, ExactOneToken, TheOneToken); 1180 if (ExactOneToken) 1181 { 1182 output.write("Tok == ", TheOneToken); 1183 } 1184 else 1185 { 1186 output.write("SetT["); 1187 output.write(DIV(Nont[N].FollowIndex - firstGenSetT, nElemsPerSET)); 1188 output.write("][Tok] & 1uL << "); 1189 output.write(MOD(Nont[N].FollowIndex - firstGenSetT, nElemsPerSET)); 1190 } 1191 output.writeln(" || IsRepairMode)"); 1192 output.writeln("{"); 1193 EvalGen.InitScope(opt.Scope); 1194 EvalGen.GenAnalPred(N, opt.Formal.Params); 1195 EvalGen.GenSynPred(N, opt.Formal.Params); 1196 output.writeln("break;"); 1197 output.writeln("}"); 1198 output.write("ErrorRecovery("); 1199 output.write(Nont[N].OptExp - firstGenSet); 1200 output.write(", "); 1201 output.write(Nont[N].OptRec - firstGenSet); 1202 output.writeln(");"); 1203 output.writeln("}"); 1204 } 1205 else if (cast(EAG.Rep) EAG.HNont[N].Def) 1206 { 1207 if (Poss <= Nont[N].Follow && (Nont[N].First & Poss).bitsSet.empty) 1208 { 1209 warn!"dead braces in %s\n%s"(EAG.NamedHNontRepr(N), EAG.HNont[N].Def.Sub.Pos); 1210 Warning = true; 1211 } 1212 EvalGen.GenRepStart(N.to!int); 1213 output.writeln("while (1)"); 1214 output.writeln("{"); 1215 output.write("if ("); 1216 TestOneToken(Nont[N].First, ExactOneToken, TheOneToken); 1217 if (ExactOneToken) 1218 { 1219 output.write("Tok == ", TheOneToken); 1220 } 1221 else 1222 { 1223 output.write("SetT["); 1224 output.write(DIV(Nont[N].FirstIndex - firstGenSetT, nElemsPerSET)); 1225 output.write("][Tok] & 1uL << "); 1226 output.write(MOD(Nont[N].FirstIndex - firstGenSetT, nElemsPerSET)); 1227 } 1228 output.writeln(")"); 1229 output.writeln("{"); 1230 TraverseAlts(EAG.HNont[N].Def.Sub, FirstNontCall, Nont[N].First); 1231 output.writeln("}"); 1232 output.write("else if ("); 1233 TestOneToken(Nont[N].Follow, ExactOneToken, TheOneToken); 1234 if (ExactOneToken) 1235 { 1236 output.write("Tok == ", TheOneToken); 1237 } 1238 else 1239 { 1240 output.write("SetT["); 1241 output.write(DIV(Nont[N].FollowIndex - firstGenSetT, nElemsPerSET)); 1242 output.write("][Tok] & 1uL << "); 1243 output.write(MOD(Nont[N].FollowIndex - firstGenSetT, nElemsPerSET)); 1244 } 1245 output.writeln(" || IsRepairMode)"); 1246 output.writeln("break;"); 1247 output.writeln("else"); 1248 output.writeln("ErrorRecovery(", Nont[N].OptExp - firstGenSet, ", ", Nont[N].OptRec - firstGenSet, ");"); 1249 output.writeln("}"); 1250 EvalGen.GenRepEnd(N.to!int); 1251 } 1252 else 1253 { 1254 TraverseAlts(EAG.HNont[N].Def.Sub, FirstNontCall, Poss); 1255 } 1256 } 1257 1258 void WriteTab(string name) 1259 { 1260 const magicNumber = 827_092_037; 1261 size_t m; 1262 File Tab = File(settings.path(name), "w"); 1263 1264 Tab.writefln!"long %s"(magicNumber); 1265 Tab.writefln!"long %s"(TabTimeStamp); 1266 Tab.writefln!"long %s"(nElemsPerSET); 1267 Tab.writefln!"set %s"(0b10110010_01000100_00111000_11011001); 1268 for (size_t i = firstGenSetT; i < NextGenSetT; i += nElemsPerSET) 1269 { 1270 if (nElemsPerSET <= NextGenSetT - i) 1271 m = nElemsPerSET; 1272 else 1273 m = NextGenSetT - i; 1274 for (int Tok = 0; Tok < nToks; ++Tok) 1275 { 1276 size_t s = 0; 1277 1278 for (size_t j = 0; j < m; ++j) 1279 if (GenSetT[i + j][Tok]) 1280 s |= 1uL << j; 1281 Tab.writefln!"set %s"(s); 1282 } 1283 } 1284 for (size_t i = firstGenSet; i < NextGenSet; ++i) 1285 { 1286 const data = cast(size_t[]) GenSet[i]; 1287 1288 for (int j = 0; j < GenSet[i].dim; ++j) 1289 Tab.writefln!"set %s"(data[j]); 1290 } 1291 Tab.writefln!"long %s"(magicNumber); 1292 } 1293 1294 void InclFix(char Term) 1295 { 1296 import std.exception : enforce; 1297 1298 char c = Fix.front.to!char; 1299 1300 while (c != Term) 1301 { 1302 enforce(c != 0, 1303 "error: unexpected end of eELL1Gen.fix.d"); 1304 1305 output.write(c); 1306 Fix.popFront; 1307 c = Fix.front.to!char; 1308 } 1309 Fix.popFront; 1310 } 1311 1312 enum fixName = "ell1gen.fix.d"; 1313 const fileName = settings.path(EAG.BaseName ~ ".d"); 1314 1315 AllToks = BitArray(); 1316 AllToks.length = nToks + 1; 1317 Fix = Input(fixName, import(fixName)); 1318 output = File(fileName, "w"); 1319 if (parsePass) 1320 EvalGen.InitGen(output, EvalGen.parsePass, settings); 1321 else 1322 EvalGen.InitGen(output, EvalGen.onePass, settings); 1323 InclFix('$'); 1324 output.write(EAG.BaseName); 1325 InclFix('$'); 1326 name = EAG.BaseName ~ "Lex"; 1327 output.write(name); 1328 if (parsePass) 1329 { 1330 output.write(", Eval = "); 1331 output.write(EAG.BaseName); 1332 output.write("Eval"); 1333 } 1334 InclFix('$'); 1335 output.write(nToks); 1336 InclFix('$'); 1337 output.write(AllToks.dim); 1338 InclFix('$'); 1339 output.write(DIV(NextGenSetT - firstGenSetT - 1, nElemsPerSET) + 1); 1340 InclFix('$'); 1341 output.write(NextGenSet - firstGenSet); 1342 InclFix('$'); 1343 EvalGen.GenDeclarations(settings); 1344 EvalGen.GenPredProcs; 1345 InclFix('$'); 1346 TabTimeStamp = MonoTime.currTime.ticks; 1347 output.write(TabTimeStamp); 1348 InclFix('$'); 1349 AllToks[] = false; 1350 for (Tok = 0; Tok < nToks; ++Tok) 1351 AllToks[Tok] = true; // TODO: opSliceAssign 1352 foreach (N; GenNonts.bitsSet) 1353 { 1354 if (!Nont[N].Anonym) 1355 { 1356 loopCount = 0; 1357 EvalGen.ComputeVarNames(N, Yes.embed); 1358 output.write("void P", N); 1359 EvalGen.GenFormalParams(N, Yes.parNeeded); 1360 output.writeln(" // ", EAG.HNontRepr(N)); 1361 output.writeln("{"); 1362 EvalGen.GenVarDecl(N); 1363 TraverseNont(N, true, AllToks); 1364 output.writeln("}"); 1365 output.writeln; 1366 } 1367 } 1368 if (!parsePass) 1369 EmitGen.GenEmitProc(output, settings); 1370 InclFix('$'); 1371 if (parsePass) 1372 output.write("& Eval.EvalInitSucceeds()"); 1373 InclFix('$'); 1374 output.write(EAG.BaseName); 1375 InclFix('$'); 1376 output.write("P"); 1377 output.write(EAG.StartSym); 1378 InclFix('$'); 1379 if (parsePass) 1380 { 1381 output.writeln("Eval.TraverseSyntaxTree(Heap, PosHeap, ErrorCounter, V1, arityConst, info_, write);"); 1382 output.writeln("if (info_)"); 1383 output.writeln("{"); 1384 output.writeln(`stdout.write(" syntax tree uses twice ");`); 1385 output.writeln("stdout.write(NextHeap);"); 1386 output.writeln("stdout.writeln;"); 1387 output.write("}"); 1388 } 1389 else 1390 { 1391 output.writeln("if (ErrorCounter > 0)"); 1392 output.writeln("{"); 1393 output.writeln("import core.stdc.stdlib : exit, EXIT_FAILURE;"); 1394 output.writeln(`info!"errors detected: %s"(ErrorCounter);`); 1395 output.writeln("exit(EXIT_FAILURE);"); 1396 output.writeln("}"); 1397 output.writeln("else"); 1398 output.writeln("{"); 1399 EmitGen.GenEmitCall(output, settings); 1400 output.writeln("}"); 1401 EmitGen.GenShowHeap(output); 1402 } 1403 InclFix('$'); 1404 output.write(EAG.BaseName); 1405 InclFix('$'); 1406 name = EAG.BaseName ~ ".Tab"; 1407 output.write(name); 1408 InclFix('$'); 1409 output.write(EAG.BaseName); 1410 InclFix('$'); 1411 name = EAG.BaseName ~ ".Tab"; 1412 WriteTab(name); 1413 EvalGen.FinitGen; 1414 output.close; 1415 return fileName; 1416 } 1417 1418 private void NewEdge(size_t From, int To) nothrow @safe 1419 { 1420 if (NextEdge == Edge.length) 1421 Expand; 1422 Edge[NextEdge].Dest = To; 1423 Edge[NextEdge].Next = Nont[From].Edge; 1424 Nont[From].Edge = NextEdge; 1425 ++NextEdge; 1426 } 1427 1428 private void Expand() nothrow @safe 1429 { 1430 size_t ExpLen(size_t ArrayLen) 1431 { 1432 assert(ArrayLen <= DIV(size_t.max, 2)); 1433 1434 return 2 * ArrayLen; 1435 } 1436 1437 if (NextEdge >= Edge.length) 1438 { 1439 auto Edge1 = new EdgeRecord[ExpLen(Edge.length)]; 1440 1441 for (size_t i = firstEdge; i < Edge.length; ++i) 1442 Edge1[i] = Edge[i]; 1443 Edge = Edge1; 1444 } 1445 if (NextGenSet >= GenSet.length) 1446 { 1447 auto GenSet1 = new BitArray[ExpLen(GenSet.length)]; 1448 1449 for (size_t i = firstGenSet; i < GenSet.length; ++i) 1450 GenSet1[i] = GenSet[i]; 1451 GenSet = GenSet1; 1452 } 1453 if (NextGenSetT >= GenSetT.length) 1454 { 1455 auto GenSetT1 = new BitArray[ExpLen(GenSetT.length)]; 1456 1457 for (size_t i = firstGenSetT; i < GenSetT.length; ++i) 1458 GenSetT1[i] = GenSetT[i]; 1459 GenSetT = GenSetT1; 1460 } 1461 }