ngen: workaround for HW bug with cross-pipe cmod WAR hazards

third_party/ngen/ngen_asm.hpp

@@ -255,6 +255,7 @@ struct AsmInstruction {
     inline unsigned src1Typecode() const { return getTypecode(src[1]); }
     inline autoswsb::DestinationMask destinations(int &jip, int &uip) const;
     inline bool getOperandRegion(autoswsb::DependencyRegion &region, int opNum) const;
+    inline bool getCModDepRegion(autoswsb::DependencyRegion &region) const;
 
     void shiftJIP(int32_t shift) const {}
     void shiftUIP(int32_t shift) const {}
@@ -442,6 +443,15 @@ bool AsmInstruction::getOperandRegion(autoswsb::DependencyRegion &region, int op
     return true;
 }
 
+bool AsmInstruction::getCModDepRegion(autoswsb::DependencyRegion &region) const
+{
+    if (mod.getCMod() == ConditionModifier::none)
+        return false;
+
+    region = autoswsb::DependencyRegion(region.hw, 1, mod.getFlagReg());
+    return true;
+}
+
 #if defined(NGEN_GLOBAL_REGS) && !defined(NGEN_GLOBAL_REGS_DEFINED)
 #include "ngen_registers.hpp"
 #endif

third_party/ngen/ngen_auto_swsb.hpp

@@ -598,7 +598,7 @@ inline bool bboxContains(const DependencyRegion &dep1, const DependencyRegion &d
 // Check if an ARF type needs SWSB tracking.
 inline bool trackableARF(ARFType type)
 {
-    return (type == ARFType::acc || type == ARFType::a || type == ARFType::s);
+    return (type == ARFType::acc || type == ARFType::a || type == ARFType::s || type == ARFType::f);
 }
 
 // Distance in an in-order pipe after which a dependency can be ignored.
@@ -1871,6 +1871,7 @@ inline void analyze(HW hw, int tokens, Program &program, BasicBlock &bb, int pha
     std::array<int32_t, NPipes> counters;
     std::vector<Producer> depList, depListIncoming, chainProducers, pvcWARWADeps;
     std::vector<std::pair<bool, const DependencyRegion*>> depOperands;
+    DependencyRegion cmodDepRegion(hw);
 
     auto allTokens = uint32_t((uint64_t(1) << tokens) - 1);
 
@@ -2078,6 +2079,10 @@ inline void analyze(HW hw, int tokens, Program &program, BasicBlock &bb, int pha
                 depOperands.push_back(std::make_pair(rw, &regions[srcN + 1]));
             }
 
+            // Handle HW bug with cross-pipe flag register dependencies.
+            if (hw >= HW::XeHPC && insn.getCModDepRegion(cmodDepRegion))
+                depOperands.push_back(std::make_pair(true, &cmodDepRegion));
+
             // Handle PVC HW bug with WAR dependencies on send instructions.
             auto pww = analyzePVCWARWA(hw, program, bb, phase, consumeOp, pvcWARWADeps);
 

third_party/ngen/ngen_gen12.hpp

@@ -453,6 +453,8 @@ struct Instruction12 {
     inline autoswsb::DestinationMask destinations(int &jip, int &uip) const;
     template <typename Tag = EncodingTag12>
     inline bool getOperandRegion(autoswsb::DependencyRegion &region, int opNum) const;
+    template <typename Tag = EncodingTag12>
+    inline bool getCModDepRegion(autoswsb::DependencyRegion &region) const;
     inline bool getImm32(uint32_t &imm) const;
     inline bool getSendDesc(MessageDescriptor &desc) const;
     inline bool getARFType(ARFType &arfType, int opNum, HW hw) const;
@@ -479,6 +481,11 @@ struct InstructionXeHPC : public Instruction12 {
         return Instruction12::getOperandRegion<EncodingTagXeHPC>(region, opNum);
     }
 
+    template <typename Tag = EncodingTagXeHPC>
+    bool getCModDepRegion(autoswsb::DependencyRegion &region) const {
+        return Instruction12::getCModDepRegion<EncodingTagXeHPC>(region);
+    }
+
     bool eot() const {
         return Instruction12::eot();
     }
@@ -1086,6 +1093,45 @@ bool Instruction12::getOperandRegion(autoswsb::DependencyRegion &region, int opN
     return true;
 }
 
+template <typename Tag>
+bool Instruction12::getCModDepRegion(autoswsb::DependencyRegion &region) const
+{
+    /* Skip instructions with no cmod fields */
+    auto op = opcode();
+    switch (op) {
+        case Opcode::directive:
+        case Opcode::sync:
+        case Opcode::illegal:
+        case Opcode::bfn:
+        case Opcode::send:
+        case Opcode::sendc:
+        case Opcode::dpas:
+        case Opcode::dpasw:
+        case Opcode::math:
+            return false;
+        default:
+            if (isBranch(op))
+                return false;
+            break;
+    }
+
+    auto cmod = static_cast<ConditionModifier>(binary.cmod);
+    if (cmod == ConditionModifier::none)
+        return false;
+
+    /* Decode cmod flag */
+    constexpr bool xeHPC = !std::is_same<Tag, EncodingTag12>::value;
+    unsigned flagn = xeHPC ? commonXeHPC.flagReg : common.flagReg;
+    unsigned lg2ES = xeHPC ? commonXeHPC.execSize : common.execSize;
+
+    auto flag = FlagRegister::createFromIndex(flagn);
+    if (lg2ES >= 5)
+        flag.setType(DataType::ud);
+    region = autoswsb::DependencyRegion(region.hw, 1, flag);
+
+    return true;
+}
+
 unsigned Instruction12::srcTypecode(int opNum) const
 {
     auto op = opcode();